import scrapy

from ..items import LiepinItem
from urllib import parse


class LiePin(scrapy.Spider):
    name = "liepin"

    base_url = "https://www.liepin.com/zhaopin/?headId=ec4574d21ab227decedad29bb952a5c9&ckId=y0igzuf4rcdczu6sizc062yol8li5vbb&oldCkId=ec4574d21ab227decedad29bb952a5c9&fkId=hj48danlk9w2cbl3tliy5br368hk1oya&skId=hj48danlk9w2cbl3tliy5br368hk1oya&sfrom=search_job_pc&key={}&currentPage={}&scene=page"

    def start_requests(self):

        words = ["大数据", "hadoop", "spark", "flink", "数据处理", "数据开发"]

        for word in words:

            # 循环获取多个页面
            for page in range(100):
                url = self.base_url.format(parse.quote(word), page)

                yield scrapy.Request(url=url)

    def parse(self, response):
        # 解析数据
        for job in response.xpath("//div[@class='job-list-item']"):
            # 岗位名
            job_name = job.xpath(".//div[@class='ellipsis-1']/text()").extract()[0]
            # 工作地点
            location = job.xpath(".//span[@class='ellipsis-1']/text()").extract()[0]
            # 薪资范围
            job_salary = job.xpath(".//span[@class='job-salary']/text()").extract()[0]

            labels_tag = job.xpath(".//span[@class='labels-tag']/text()").extract()

            # 年限
            years = labels_tag[0]

            # 学历
            education = labels_tag[1]

            # 技能
            tag = labels_tag[2:]

            # 公司
            company_name = job.xpath(".//span[@class='company-name ellipsis-1']/text()").extract()[0]

            company_tags_box = job.xpath(".//div[@class='company-tags-box ellipsis-1']/span/text()").extract()

            # 行业
            hangye = ""
            # 融资
            rongzi = "未知"
            # 公司规模
            guimo = ""

            if len(company_tags_box) == 2:
                hangye = company_tags_box[0]
                guimo = company_tags_box[1]

            if len(company_tags_box) == 3:
                hangye = company_tags_box[0]
                rongzi = company_tags_box[1]
                guimo = company_tags_box[2]

            recruiter_name = job.xpath(".//div[@class='recruiter-name ellipsis-1']/text()").extract()[0]
            recruiter_title = job.xpath(".//div[@class='recruiter-title ellipsis-1']/text()").extract()
            if len(recruiter_title) != 0:
                recruiter_title = recruiter_title[0]
            else:
                recruiter_title = ""

            # 返回数据
            yield LiepinItem(job_name=self.parse_data(job_name),
                             location=self.parse_data(location),
                             job_salary=self.parse_data(job_salary),
                             years=self.parse_data(years),
                             education=self.parse_data(education),
                             tag=self.parse_data(tag),
                             company_name=self.parse_data(company_name),
                             hangye=self.parse_data(hangye),
                             rongzi=self.parse_data(rongzi),
                             guimo=self.parse_data(guimo),
                             recruiter_name=self.parse_data(recruiter_name),
                             recruiter_title=self.parse_data(recruiter_title))

    def parse_data(self, data):

        if isinstance(data, str):
            return data.replace("\t", "").replace("\n", "").replace("\r", "")

        if isinstance(data, list):
            # join 将一个列表拼接成一个字符串
            return ",".join(data)
